
import sys
!{sys.executable} -m pip install numpy pandas matplotlib scikit-learn | grep -v 'already satisfied'

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt


# Read the dataset
# I used only the small dataset because the computing time for the selected ML model including all chosen features took about 45 minutes to compute. 
# When using the test set in the end, the computing time was significantly lower - so I guess that the computing time is exponentially increasing the larger the data set is.
flights = pd.read_csv("Combined_Flights_small.csv")

# Clear the maximum number of columns to be displayed, so that all will be visible.
pd.set_option('display.max_columns', None)

# Take a first look at the data
flights.head()


# How many rows and columns/features does the dataset have?
flights.shape

# 29,191 rows (flighst) and 47 columns/features (using the small dataset)

(29191, 47)


# stratified sampling

from sklearn.model_selection import StratifiedShuffleSplit

# define how to split the data
shuffled_data = StratifiedShuffleSplit(
        n_splits=1,
        test_size=0.2,
        random_state=5059 # set seed to maintain split
)

# create indexes for train and test sets
# we want a stratified sample based on year, month and disruption 
# (disruption is important because it will be our label; year and month is important because we know that there are differences due to COVID19)
[(train_index, test_index)] = shuffled_data.split(
    flights,
    flights[["Year", "Month", "Disruption"]]
)

# create train and test sets
flights_train = flights.loc[train_index]
flights_test = flights.loc[test_index]


# print info to detect null values and inspect data types
flights_train.info()

<class 'pandas.core.frame.DataFrame'>
Index: 23352 entries, 3664 to 993
Data columns (total 47 columns):
 #   Column                                   Non-Null Count  Dtype  
---  ------                                   --------------  -----  
 0   Unnamed: 0                               23352 non-null  int64  
 1   FlightDate                               23352 non-null  object 
 2   Airline                                  23352 non-null  object 
 3   Origin                                   23352 non-null  object 
 4   Dest                                     23352 non-null  object 
 5   Cancelled                                23352 non-null  int64  
 6   Diverted                                 23352 non-null  int64  
 7   CRSDepTime                               23352 non-null  int64  
 8   CRSElapsedTime                           23352 non-null  int64  
 9   Distance                                 23352 non-null  int64  
 10  Year                                     23352 non-null  int64  
 11  Quarter                                  23352 non-null  int64  
 12  Month                                    23352 non-null  int64  
 13  DayofMonth                               23352 non-null  int64  
 14  DayOfWeek                                23352 non-null  int64  
 15  Marketing_Airline_Network                23352 non-null  object 
 16  Operated_or_Branded_Code_Share_Partners  23352 non-null  object 
 17  DOT_ID_Marketing_Airline                 23352 non-null  int64  
 18  IATA_Code_Marketing_Airline              23352 non-null  object 
 19  Flight_Number_Marketing_Airline          23352 non-null  int64  
 20  Operating_Airline                        23352 non-null  object 
 21  DOT_ID_Operating_Airline                 23352 non-null  int64  
 22  IATA_Code_Operating_Airline              23352 non-null  object 
 23  Tail_Number                              23134 non-null  object 
 24  Flight_Number_Operating_Airline          23352 non-null  int64  
 25  OriginAirportID                          23352 non-null  int64  
 26  OriginAirportSeqID                       23352 non-null  int64  
 27  OriginCityMarketID                       23352 non-null  int64  
 28  OriginCityName                           23352 non-null  object 
 29  OriginState                              23352 non-null  object 
 30  OriginStateFips                          23352 non-null  int64  
 31  OriginStateName                          23352 non-null  object 
 32  OriginWac                                23352 non-null  int64  
 33  DestAirportID                            23352 non-null  int64  
 34  DestAirportSeqID                         23352 non-null  int64  
 35  DestCityMarketID                         23352 non-null  int64  
 36  DestCityName                             23352 non-null  object 
 37  DestState                                23352 non-null  object 
 38  DestStateFips                            23352 non-null  int64  
 39  DestStateName                            23352 non-null  object 
 40  DestWac                                  23352 non-null  int64  
 41  DepTimeBlk                               23352 non-null  object 
 42  CRSArrTime                               23352 non-null  int64  
 43  ArrDel15                                 22680 non-null  float64
 44  ArrTimeBlk                               23352 non-null  object 
 45  DistanceGroup                            23352 non-null  int64  
 46  Disruption                               23352 non-null  int64  
dtypes: float64(1), int64(28), object(18)
memory usage: 8.6+ MB


# inspect how many unique/distinct values there are per feature
flights_train.nunique()

Unnamed: 0                                 23352
FlightDate                                  1673
Airline                                       28
Origin                                       364
Dest                                         350
Cancelled                                      2
Diverted                                       2
CRSDepTime                                  1154
CRSElapsedTime                               413
Distance                                    1445
Year                                           5
Quarter                                        4
Month                                         12
DayofMonth                                    31
DayOfWeek                                      7
Marketing_Airline_Network                     11
Operated_or_Branded_Code_Share_Partners       16
DOT_ID_Marketing_Airline                      11
IATA_Code_Marketing_Airline                   11
Flight_Number_Marketing_Airline             6079
Operating_Airline                             28
DOT_ID_Operating_Airline                      28
IATA_Code_Operating_Airline                   28
Tail_Number                                 6007
Flight_Number_Operating_Airline             6080
OriginAirportID                              364
OriginAirportSeqID                           390
OriginCityMarketID                           339
OriginCityName                               358
OriginState                                   52
OriginStateFips                               52
OriginStateName                               52
OriginWac                                     52
DestAirportID                                350
DestAirportSeqID                             377
DestCityMarketID                             325
DestCityName                                 344
DestState                                     52
DestStateFips                                 52
DestStateName                                 52
DestWac                                       52
DepTimeBlk                                    19
CRSArrTime                                  1244
ArrDel15                                       2
ArrTimeBlk                                    19
DistanceGroup                                 11
Disruption                                     2
dtype: int64


# Have a quick look at the data
%matplotlib inline
flights_train.hist(bins=70, figsize=(20,15))
plt.show()


# Airline
# Categories not balanced - some airlines operated more flights than others.
# We could group the airlines into large, medium, and small airline, by the number of flights operated.

# Count the number of flights per airline
flight_count_airline = flights_train.groupby(["Airline"]).size()

# Plot bar chart
flight_count_airline.plot(kind="bar", figsize=(20,5))
plt.title("Count of Flights per Airline")
plt.show()


# OriginState
# Origin not balanced - some origin states are more often frequented than others
flights_train["OriginState"].hist(bins=120, figsize=(20,5))

plt.title("Count of flights per Origin State")

Text(0.5, 1.0, 'Count of flights per Origin State')


# DestState
# as for OriginState: categories not balanced, some destinations are more frequent than others
flights_train["DestState"].hist(bins=120, figsize=(20,5))

plt.title("Count of flights per Destination State")

Text(0.5, 1.0, 'Count of flights per Destination State')


# Categories are not very balanced; 
# 19 distinct categories. We could consider splitting time slots into before noon and after noon to reduce dimensions.
flights_train["DepTimeBlk"].hist(bins=60, figsize=(20,5));
plt.xticks(rotation=45)
plt.title("Count of flights per Departure Time Block")

Text(0.5, 1.0, 'Count of flights per Departure Time Block')


# There is no obvious pattern compared to the order of the data for Disruption
sns.scatterplot(x=list(range(len(flights_train))), y=flights_train["Disruption"])

<Axes: ylabel='Disruption'>


# There is no obvious pattern compared to the order of the data for Year
# checked also for Month, DayofMonth
sns.scatterplot(y=list(range(len(flights_train))), x=flights_train["Year"])

<Axes: xlabel='Year'>


# Distance
sns.scatterplot(x=flights_train["Distance"], y=flights_train["Disruption"])

# flights between 4000 and 5000 miles distance seem not to be disrupted

<Axes: xlabel='Distance', ylabel='Disruption'>


# CRSDepTime
sns.scatterplot(x=flights_train["CRSDepTime"], y=flights_train["Disruption"])

<Axes: xlabel='CRSDepTime', ylabel='Disruption'>


# CRSArrTime
sns.scatterplot(x=flights_train["CRSArrTime"], y=flights_train["Disruption"])

<Axes: xlabel='CRSArrTime', ylabel='Disruption'>


# Count the number of flights per year
flight_count_airline = flights_train.groupby(["Year"]).size()

# Plot bar chart
flight_count_airline.plot(kind="bar")
plt.title("Count of Flights per Year")
plt.show()
# We have a different total number of flights per year.


# Count number of disruptions for each year
disruptions = flights_train.groupby("Year")["Disruption"].value_counts().unstack()

# Calculate total number of flights for each year
total_flights = flights_train["Year"].value_counts()

# Calculate ratio of disruptions per count of total flights per year
disruption_ratio = disruptions.divide(total_flights, axis=0)

# Define colors for the stacked bars
colors = ['olive', 'darkorange']

# Plot the bar chart
disruption_ratio.plot(kind="bar", stacked=True, color=colors)
plt.title("Ratio of Disrupted and Non-Disrupted Flights per Year")
plt.legend(labels=['Not Disrupted', 'Disrupted'])
plt.xlabel("Year")
plt.ylabel("Ratio")
plt.show()

# Calculate the ratio of disruptions per count of total flights per year
disruption_ratio = disruptions[1].divide(total_flights, axis=0)

# Plot the line graph for disrupted ratio
disruption_ratio.plot(kind="line", marker="o", color="red")
plt.title("Ratio of Disrupted Flights per Total Flights by Year")
plt.xlabel("Year")
plt.ylabel("Disruption Ratio")
plt.grid(True)
plt.show()

# We can see that despite the COVID19 pandemic, there are less disruptions in 2020 and 2021 compared to the other years. 
# However, we can see that 2022 (post pandemic) has more disruptions than pre pandemic years.


# Combine Year and Quarter into a single column
flights_train["Year_Quarter"] = flights_train["Year"].astype(str) + "-" + flights_train["Quarter"].astype(str)

# Count the number of disruptions for each year and quarter
disruptions = flights_train.groupby(["Year_Quarter", "Disruption"]).size().unstack()

# Calculate the total number of flights for each year and quarter
total_flights = flights_train["Year_Quarter"].value_counts()

# Calculate the ratio of disruptions per count of total flights per year and quarter
disruption_ratio = disruptions[1].divide(total_flights, axis=0)

# Plot the line graph for disrupted ratio
disruption_ratio.plot(figsize=(15, 5), kind="line", marker="o", color="r")
plt.title("Ratio of Disrupted Flights per Total Flights by Year and Quarter")
plt.xlabel("Year-Quarter")
plt.ylabel("Disruption Ratio")
plt.xticks(rotation=45)
plt.grid(True)
plt.show()

# A more granular exploration per year and quarter reveals that there were significantly more disruptions in the second 2020 quarter - during the first pandemic phase,
# followed by a significant drop of the disruption ratio in the third 2020 quarter. After that, the disruption rate rises and exceeds the pre pandemic level.


# Count the number of disruptions for each month
disruptions = flights_train.groupby(["Month", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(kind="bar", stacked=True, color=colors)
plt.title('Disruption by Month')
plt.legend(labels=['Not Disrupted', 'Disrupted'])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per month
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(kind="line", marker="o", color='red')
plt.title('Ratio of Disrupted to Non-Disrupted Flights per Month')
plt.xlabel('Month')
plt.ylabel('Ratio')
plt.show()

# We see differences between months. Some month have a higher disruption rate than others.


# Count the number of disruptions for each quarter
disruptions = flights_train.groupby(["Quarter", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(kind="bar", stacked=True, color=colors)
plt.title("Disruption by Quarter")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per quarter
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(kind="line", marker="o", color='red')
plt.title('Ratio of Disrupted to Non-Disrupted Flights per Quarter')
plt.xlabel('Quarter')
plt.ylabel('Ratio')
plt.show()


# Count the number of disruptions for each day of a week
disruptions = flights_train.groupby(["DayOfWeek", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(kind="bar", stacked=True, color=colors)
plt.title("Disruption by Day of Week")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per day of the week
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(kind="line", marker="o", color='red')
plt.title('Ratio of Disrupted to Non-Disrupted Flights per Day of the Week')
plt.xlabel('Day of Week')
plt.ylabel('Ratio')
plt.show()


# Count the number of disruptions for each day of a month
disruptions = flights_train.groupby(["DayofMonth", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(kind="bar", stacked=True, color=colors)
plt.title("Disruption by Day of Month")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per month
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(kind="line", marker="o", color='red')
plt.title('Ratio of Disrupted to Non-Disrupted Flights per Day of the Month')
plt.xlabel('Day of Month')
plt.ylabel('Ratio')
plt.show()


# Convert the column to datetime data type
flights_train["FlightDate"] = pd.to_datetime(flights_train["FlightDate"])

# Filter the DataFrame to include only rows where 'Disruption' is equal to 1
disruptions = flights_train[flights_train["Disruption"] == 1]

# Group by the date column and count the number of disruptions for each date
disruptions_count = disruptions.groupby("FlightDate").size()

# Plot the timeline
plt.figure(figsize=(30, 6))
plt.plot(disruptions_count.index, disruptions_count.values, color="orange")
plt.ylabel("Count of Disruptions")
plt.title("Timeline of Disruptions")
plt.grid(True)
plt.show()


# Convert the column to datetime data type
flights_train["FlightDate"] = pd.to_datetime(flights_train["FlightDate"])

# Group by the date column and count the number of flights for each date
flights_count = flights_train.groupby("FlightDate").size()

# Plot the timeline
plt.figure(figsize=(30, 6))
plt.plot(flights_count.index, flights_count.values)
plt.ylabel("Count of Flights")
plt.title("Timeline of Flights")
plt.grid(True)
plt.show()


# Calculate the ratio of disruptions to flights for each date
ratio = disruptions_count.div(flights_count)

# Plot the timeline
plt.figure(figsize=(30, 6))
plt.plot(ratio.index, ratio.values, color="red")
plt.ylabel("Ratio of Disruptions to Flights")
plt.title("Timeline of Disruption Ratio")
plt.grid(True)
plt.show()


# Count the number of disruptions for each airline
disruptions = flights_train.groupby(["Airline", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(kind="bar", stacked=True, color=colors)
plt.title("Disruption by Airline")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per month
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(kind="bar", color='red')
plt.title('Ratio of Disrupted to Non-Disrupted Flights per Airline')
plt.xlabel('Airline')
plt.ylabel('Ratio')
plt.show()

# seems like Airline could be a weak indicator for disruptions - we see e.g. Delta has lower disruptions than American Airlines despite more flights in total


# Count the number of disruptions for code sharing flights
disruptions = flights_train.groupby(["Operated_or_Branded_Code_Share_Partners", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(kind="bar", stacked=True, color=colors)
plt.title("Disruption by Operated_or_Branded_Code_Share_Partners")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per month
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(kind="bar", color='red')
plt.title('Ratio of Disrupted to Non-Disrupted Flights per Code Share Partners')
plt.xlabel('Operated_or_Branded_Code_Share_Partners')
plt.ylabel('Ratio')
plt.show()

# There seem to be significant differences between code share categories. Some partners have much higher disruption ratios than others.


# Count the number of disruptions for each airline
disruptions = flights_train.groupby(["OriginState", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(figsize=(20,5), kind="bar", stacked=True, color=colors)
plt.title("Disruption by OriginState")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per month
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(figsize=(20,5), kind="bar", color='red')
plt.title('Ratio of Disrupted to Non-Disrupted Flights per OriginState')
plt.xlabel('OriginState')
plt.ylabel('Ratio')
plt.show()


# Count the number of disruptions for each destination
disruptions = flights_train.groupby(["DestState", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(figsize=(20,5), kind="bar", stacked=True, color=colors)
plt.title("Disruption by DestState")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per month
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(figsize=(20,5), kind="bar", color='red')
plt.title('Ratio of Disrupted to Non-Disrupted Flights per DestState')
plt.xlabel('DestState')
plt.ylabel('Ratio')
plt.show()


# Count the number of disruptions for each flight number
disruptions = flights_train.groupby(["Flight_Number_Operating_Airline", "Disruption"]).size().unstack()

# Select top N categories based on frequency counts
top_n = 100
top_disruptions = disruptions.sum(axis=1).nlargest(top_n).index
disruptions_top_n = disruptions.loc[top_disruptions]

# Plot the grouped bar chart
disruptions_top_n.plot(figsize=(20,5), kind="bar", stacked=True, color=colors)
plt.title("Disruption by Top 100 Flight Numbers")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Total number of flights for each flight number
total_flights = disruptions_top_n.sum(axis=1)

# Ratio of disrupted to total flights for each flight number
disruption_ratio = disruptions_top_n[1] / total_flights

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(figsize=(20,5), kind="bar", color='red')
plt.title('Disruption Ratio for Top Flight Numbers')
plt.xlabel('Flight Number')
plt.ylabel('Ratio')
plt.show()

# flight number may be a predictor for disruption
# there seem to be differences between flight numbers: Some were never disrupted, some very often.


# Count the number of disruptions for each distance group
disruptions = flights_train.groupby(["DistanceGroup", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(kind="bar", stacked=True, color=colors)
plt.title("Disruption by DistanceGroup")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per month
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(kind="line", color='red', marker="o")
plt.title('Ratio of Disrupted to Non-Disrupted Flights per DistanceGroup')
plt.xlabel('DistanceGroup')
plt.ylabel('Ratio')
plt.show()

# There are some differences, the disruption rate is between 0.2 and 0.3 across all distances. 
# There is no clear trend, e.g. that long distance flights are less disrupted than short distance flights. 
# As there is no clear pattern visible, I consider the distance in general not as a good predictor for disruptions.


# Count the number of disruptions for each departure slot
disruptions = flights_train.groupby(["DepTimeBlk", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(kind="bar", stacked=True, color=colors)
plt.title("Disruption by DepTimeBlk")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per month
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(kind="line", color='red', marker="o")
plt.title('Ratio of Disrupted to Non-Disrupted Flights per DepTimeBlk')
plt.xlabel('DepTimeBlk')
plt.ylabel('Ratio')
plt.show()

# There are significant differences between departure time slots
# Afternoon flights seem to have an increased disruption ratio compared to flights departing in the morning or late evening/night.


# Count the number of disruptions for each arrival slot
disruptions = flights_train.groupby(["ArrTimeBlk", "Disruption"]).size().unstack()

# Plot the grouped bar chart
disruptions.plot(kind="bar", stacked=True, color=colors)
plt.title("Disruption by ArrTimeBlk")
plt.legend(labels=["Not Disrupted", "Disrupted"])
plt.show()

# Calculate the ratio of disrupted to non-disrupted flights per month
disruption_ratio = disruptions[1] / disruptions[0]

# Plot the ratio as a stacked bar chart
disruption_ratio.plot(kind="line", color='red', marker="o")
plt.title('Ratio of Disrupted to Non-Disrupted Flights per ArrTimeBlk')
plt.xlabel('ArrTimeBlk')
plt.ylabel('Ratio')
plt.show()

# We see that flights departing directly after midnight are often disrupted.
# We see that flights before noon are less likely to be disrupted, with a rising trend during the day and highest ratio of disruptions with flights arriving in the evening/at night.


# Relationship between features
cross_tab = pd.crosstab(flights_train["Airline"], flights_train["DepTimeBlk"])

# Plot heatmap
plt.figure(figsize=(12, 8))
sns.heatmap(cross_tab, cmap="YlGnBu", annot=True, fmt="d")
plt.title("Relationship between Airline and Departure Time Block")
plt.xlabel("Departure Time Block")
plt.ylabel("Airline")
plt.show()

# Relationship between features
cross_tab = pd.crosstab(flights_train["Airline"], flights_train["ArrTimeBlk"])

# Plot heatmap
plt.figure(figsize=(12, 8))
sns.heatmap(cross_tab, cmap="YlGnBu", annot=True, fmt="d")
plt.title("Relationship between Airline and Arrival Time Block")
plt.xlabel("Arrival Time Block")
plt.ylabel("Airline")
plt.show()


# Relationship between features
cross_tab = pd.crosstab(flights_train["Airline"], flights_train["DistanceGroup"])

# Plot heatmap
plt.figure(figsize=(12, 8))
sns.heatmap(cross_tab, cmap="YlGnBu", annot=True, fmt="d")
plt.title("Relationship between Airline and Distance Group")
plt.xlabel("Distance Group")
plt.ylabel("Airline")
plt.show()


# Relationship between features

# Airline and OriginState
cross_tab = pd.crosstab(flights_train["Airline"], flights_train["OriginState"])

# Plot heatmap
plt.figure(figsize=(20, 10))
sns.heatmap(cross_tab, cmap="YlGnBu", annot=True, fmt="d")
plt.title("Relationship between Airline and Origin")
plt.xlabel("Origin State")
plt.ylabel("Airline")
plt.show()

# Airline and DestState
cross_tab = pd.crosstab(flights_train["Airline"], flights_train["DestState"])

# Plot heatmap
plt.figure(figsize=(20, 10))
sns.heatmap(cross_tab, cmap="YlGnBu", annot=True, fmt="d")
plt.title("Relationship between Airline and Destination")
plt.xlabel("Dest State")
plt.ylabel("Airline")
plt.show()

# Some Airlines seem to have a primary origin state - possibly, because airlines oftentimes have something like a "home" or base airport. 
# It is difficult to visually represent the relationship between airlines and airports as there are so many different airports across the US.
# However, using state instead gives us an indication that there are indeed relationships between airline and origin/destination of flights.
# We can also see that the heatmaps for Origin and Destination look similar - which supports the theory of airlines having home airports.
# Therefore, I think that including either of origin or destination state for the prediction is sufficient, no need to include both.


# features
columns_to_keep = ["Airline", "DepTimeBlk", "ArrTimeBlk", "OriginState", "DistanceGroup", "Month", "DayOfWeek", "Flight_Number_Operating_Airline", "Operated_or_Branded_Code_Share_Partners", "Disruption"]

# "Flight_Number_Operating_Airline",

flights_train = flights_train[columns_to_keep]
flights_train.head()


# DepTimeBlk and ArrTimeBlk

# We have seen that there might be a difference in disruptions in the departure or arrival time of a flight. 
# It seemed like there are more disruptions in flights departing or arriving late.
# I will convert morning/evening departures or arrivals into 0 and afternoon into 1
# Note that for arrivals, I put the 0001-0059 time slot to the late day time period, because we saw that their disruption ratio is on a similar level than the late day arrivals.

def transformer_timeslots_binary(X):

    early_day_arrival = [
        "0600-0659",
        "0700-0759",
        "0800-0859",
        "0900-0959",
        "1000-1059",
        "1100-1159",
        "1200-1259",
        "1300-1359"          
    ]
    later_day_arrival = [
        "0001-0059",
        "1400-1459",
        "1500-1559",
        "1600-1659",
        "1700-1759",
        "1800-1859",
        "1900-1959",
        "2000-2059",
        "2100-2159",
        "2200-2259",
        "2300-2359" 
    ]
    
    early_day_departure = [
        "0001-0059",
        "0600-0659",
        "0700-0759",
        "0800-0859",
        "0900-0959",
        "1000-1059",
        "1100-1159",
        "1200-1259",
        "1300-1359"          
    ]
    later_day_departure = [
        "1400-1459",
        "1500-1559",
        "1600-1659",
        "1700-1759",
        "1800-1859",
        "1900-1959",
        "2000-2059",
        "2100-2159",
        "2200-2259",
        "2300-2359" 
    ]

    arrival_conditions = [
        (X["ArrTimeBlk"].isin(early_day_arrival)),
        (X["ArrTimeBlk"].isin(later_day_arrival)),
    ]
    
    departure_conditions = [
        (X["DepTimeBlk"].isin(early_day_departure)),
        (X["DepTimeBlk"].isin(later_day_departure)),
    ]
    
    choices = [0, 1]


    X["ArrTimeBlk"] =  np.select(arrival_conditions, choices)
    # X["ArrTimeBlk"] = pd.Categorical(X["ArrTimeBlk"])

    X["DepTimeBlk"] =  np.select(departure_conditions, choices)
    # X["DepTimeBlk"] = pd.Categorical(X["DepTimeBlk"])

    return X


# check data types for conversion
# flights_train.info()

# define for which columns we convert the data type
convert_datatype_toCategory = [
    "Airline",
    "OriginState", 
    "DistanceGroup", 
    "Month", 
    "DayOfWeek",
    "Flight_Number_Operating_Airline", 
    "Operated_or_Branded_Code_Share_Partners"
]

convert_datatype_toInt = [
    "DepTimeBlk",
    "ArrTimeBlk",
    "Disruption"
]

def convert_datatype(X, convert_datatype_toCategory, convert_datatype_toInt):

    # for every datatype in the array, change the type to categorical
    for catcol in convert_datatype_toCategory:
        X[catcol] = pd.Categorical(X[catcol])

    for catcol in convert_datatype_toInt:
        X[catcol] = X[catcol].astype(int)

    # check new data types
    flights_train.dtypes

    return X


# find columns with null values
# no null values in the chosen features
flights_train.info()

<class 'pandas.core.frame.DataFrame'>
Index: 23352 entries, 3664 to 993
Data columns (total 10 columns):
 #   Column                                   Non-Null Count  Dtype 
---  ------                                   --------------  ----- 
 0   Airline                                  23352 non-null  object
 1   DepTimeBlk                               23352 non-null  object
 2   ArrTimeBlk                               23352 non-null  object
 3   OriginState                              23352 non-null  object
 4   DistanceGroup                            23352 non-null  int64 
 5   Month                                    23352 non-null  int64 
 6   DayOfWeek                                23352 non-null  int64 
 7   Flight_Number_Operating_Airline          23352 non-null  int64 
 8   Operated_or_Branded_Code_Share_Partners  23352 non-null  object
 9   Disruption                               23352 non-null  int64 
dtypes: int64(5), object(5)
memory usage: 2.0+ MB


def onehotencoding(X):

    from sklearn.preprocessing import OneHotEncoder

    # define encoder
    one_hot_encoder = OneHotEncoder(sparse_output=False)
    one_hot_encoder.set_output(transform="pandas")

    categorical_attributes = ['Airline', 'OriginState', 'DistanceGroup', 'Month', 'DayOfWeek', 'Flight_Number_Operating_Airline', 'Operated_or_Branded_Code_Share_Partners']
    data_categorical_encoded = pd.DataFrame(one_hot_encoder.fit_transform(X[categorical_attributes]))
    # 'Flight_Number_Operating_Airline',
    X = pd.concat((X, data_categorical_encoded), axis=1)

    # drop redundant features
    columns_to_drop = ['Airline', 'OriginState', 'DistanceGroup', 'Month', 'DayOfWeek', 'Flight_Number_Operating_Airline', 'Operated_or_Branded_Code_Share_Partners']
    X = X.drop(columns=columns_to_drop)
    X.head()

    return X


def clean_and_prep(X, columns_to_keep):

    # drop columns
    X = X[columns_to_keep]

    # Feature Engineeering

    # Time slot
    X = transformer_timeslots_binary(X)

    # Data Conversion
    X = convert_datatype(X, convert_datatype_toCategory, convert_datatype_toInt)
    
    # One_Hot_Encoder
    X = onehotencoding(X)

    return X

# X = flights_train
X = flights_test

prepared_data = clean_and_prep(X, columns_to_keep)

prepared_data

/var/folders/k3/glkpl6qn5q77t9gk1w092j2w0000gn/T/ipykernel_7824/4067723223.py:71: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  X["ArrTimeBlk"] =  np.select(arrival_conditions, choices)
/var/folders/k3/glkpl6qn5q77t9gk1w092j2w0000gn/T/ipykernel_7824/4067723223.py:74: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  X["DepTimeBlk"] =  np.select(departure_conditions, choices)
/var/folders/k3/glkpl6qn5q77t9gk1w092j2w0000gn/T/ipykernel_7824/1109583602.py:25: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  X[catcol] = pd.Categorical(X[catcol])
/var/folders/k3/glkpl6qn5q77t9gk1w092j2w0000gn/T/ipykernel_7824/1109583602.py:25: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  X[catcol] = pd.Categorical(X[catcol])
/var/folders/k3/glkpl6qn5q77t9gk1w092j2w0000gn/T/ipykernel_7824/1109583602.py:25: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  X[catcol] = pd.Categorical(X[catcol])
/var/folders/k3/glkpl6qn5q77t9gk1w092j2w0000gn/T/ipykernel_7824/1109583602.py:28: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  X[catcol] = X[catcol].astype(int)
/var/folders/k3/glkpl6qn5q77t9gk1w092j2w0000gn/T/ipykernel_7824/1109583602.py:28: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  X[catcol] = X[catcol].astype(int)
/var/folders/k3/glkpl6qn5q77t9gk1w092j2w0000gn/T/ipykernel_7824/1109583602.py:28: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  X[catcol] = X[catcol].astype(int)


from sklearn.preprocessing import StandardScaler

# separate label from features
X = prepared_data.drop(["Disruption"], axis=1)
y = prepared_data["Disruption"]

# scale
scaler = StandardScaler()
X_data = scaler.fit_transform(X)
y_data = y.to_numpy()

print(X_data)
print(y_data)

[[ 1.06386086  0.85654529 -0.11711224 ... -0.37622397 -0.02618241
  -0.46388846]
 [ 1.06386086  0.85654529 -0.11711224 ... -0.37622397 -0.02618241
  -0.46388846]
 [ 1.06386086  0.85654529  8.5388168  ...  2.65799118 -0.02618241
  -0.46388846]
 ...
 [-0.93997255 -1.16748059 -0.11711224 ... -0.37622397 -0.02618241
  -0.46388846]
 [ 1.06386086  0.85654529 -0.11711224 ... -0.37622397 -0.02618241
  -0.46388846]
 [ 1.06386086  0.85654529 -0.11711224 ... -0.37622397 -0.02618241
  -0.46388846]]
[0 0 0 ... 0 1 0]


# What percentage of flights is disrupted?
disrupted_flights = ((flights_train["Disruption"] == 1).sum() / len(flights_train))
print(disrupted_flights)

# About 20% of flights in the dataset are disrupted. The categories in the label are, therefore, imbalanced, causing the models to predict every flight to be not disrupted.

0.19946899623158615


from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.utils.class_weight import compute_sample_weight

# create sample weights for the labels
# we have to do this because the labels are not balanced
sample_weights1 = compute_sample_weight(class_weight='balanced', y=y_data)

# create random forest classifier
clf = RandomForestClassifier()

# define parameter grid for GridSearchCV
# Note: the hyperparameters were fine tuned without the flight number feature. n_estimators "36" was chosen to be best without the flight number.
# However, including flight number, n_estimators "37" was chosen. If I had more time (45 minutes per computing round), I would try to fine tune n_estimators 
# for the model including flight numbers a bit more.
param_grid = [
    {"n_estimators": [35, 36, 37], "max_depth": [None, 5, 10, 20, 30], "min_samples_split": [2,3,4]}
]

# prepare grid_search with cross validation
grid_search = GridSearchCV(clf, param_grid, cv=3, scoring="accuracy", return_train_score=True)

# fit model with sample weights
grid_search.fit(X_data, y_data, sample_weight=sample_weights1)

# get the best estimator
final_clf = grid_search.best_estimator_
print(final_clf)

# get prediction
prediction = final_clf.predict(X_data)

# get accuracy
accuracy = final_clf.score(X_data, y_data)
print("Accuracy:", accuracy)

# print classification report
print("Classification Report:")
print(classification_report(y_data, prediction))

# print confusion matrix
print("Confusion Matrix:")
print(confusion_matrix(y_data, prediction))

RandomForestClassifier(n_estimators=35)
Accuracy: 0.9974310669635211
Classification Report:
              precision    recall  f1-score   support

           0       1.00      1.00      1.00      4675
           1       1.00      0.99      0.99      1164

    accuracy                           1.00      5839
   macro avg       1.00      0.99      1.00      5839
weighted avg       1.00      1.00      1.00      5839

Confusion Matrix:
[[4675    0]
 [  15 1149]]


from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix

# split data into training and validation sets
X_train, X_valid, y_train, y_valid = train_test_split(X_data, y_data, test_size=0.2, random_state=42)

# create a logistic regression model
logistic_reg = LogisticRegression()

# fit model to the training data
logistic_reg.fit(X_train, y_train)

# get prediction on the validation set
prediction = logistic_reg.predict(X_valid)

# get accuracy on validation set
accuracy = accuracy_score(y_valid, prediction)
print("Accuracy:", accuracy)

# print classification report
print("Classification Report:")
print(classification_report(y_valid, prediction))

# print confusion matrix
print("Confusion Matrix:")
print(confusion_matrix(y_valid, prediction))

Accuracy: 0.7303082191780822
Classification Report:
              precision    recall  f1-score   support

           0       0.80      0.88      0.84       933
           1       0.21      0.12      0.15       235

    accuracy                           0.73      1168
   macro avg       0.50      0.50      0.50      1168
weighted avg       0.68      0.73      0.70      1168

Confusion Matrix:
[[825 108]
 [207  28]]

/Users/lk/anaconda3/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(


from sklearn.linear_model import SGDRegressor
from sklearn.metrics import mean_squared_error

sgd_regression_model = SGDRegressor(max_iter=1000, tol=0.001, penalty=None, eta0=0.1, random_state=42)

sgd_regression_model.fit(X_train, y_train.ravel());
sgd_regression_model.intercept_, sgd_regression_model.coef_

print(sgd_regression_model)

# get prediction on the validation set
prediction = sgd_regression_model.predict(X_valid)

# calculate Mean Squared Error
mse = mean_squared_error(y_valid, prediction)
print("Mean Squared Error:", mse)

SGDRegressor(eta0=0.1, penalty=None, random_state=42)
Mean Squared Error: 7.397445399153175e+25

	Unnamed: 0	FlightDate	Airline	Origin	Dest	CRSDepTime	CRSElapsedTime	Distance	Year	Quarter	Month	DayofMonth	DayOfWeek	Marketing_Airline_Network	Operated_or_Branded_Code_Share_Partners	DOT_ID_Marketing_Airline	IATA_Code_Marketing_Airline	Flight_Number_Marketing_Airline	Operating_Airline	DOT_ID_Operating_Airline	IATA_Code_Operating_Airline	Tail_Number	Flight_Number_Operating_Airline	OriginAirportID	OriginAirportSeqID	OriginCityMarketID	OriginCityName	OriginState	OriginStateFips	OriginStateName	OriginWac	DestAirportID	DestAirportSeqID	DestCityMarketID	DestCityName	DestState	DestStateFips	DestStateName	DestWac	DepTimeBlk	CRSArrTime	ArrDel15	ArrTimeBlk	DistanceGroup	Disruption
0	1	2018-09-07	SkyWest Airlines Inc.	SLC	LAX	1953	119	590	2018	3	9	7	5	UA	UA_CODESHARE	19977	UA	5563	OO	20304	OO	N208SY	5563	14869	1486903	34614	Salt Lake City, UT	UT	49	Utah	87	12892	1289208	32575	Los Angeles, CA	CA	6	California	91	1900-1959	2052	0.0	2000-2059	3	0
1	2	2018-06-27	Air Wisconsin Airlines Corp	ORD	ILM	1355	136	760	2018	2	6	27	3	UA	UA_CODESHARE	19977	UA	4835	ZW	20046	ZW	N467AW	4835	13930	1393006	30977	Chicago, IL	IL	17	Illinois	41	12323	1232305	32323	Wilmington, NC	NC	37	North Carolina	36	1300-1359	1711	1.0	1700-1759	4	1
2	3	2018-03-14	Southwest Airlines Co.	MCO	SJU	1705	160	1189	2018	1	3	14	3	WN	WN	19393	WN	1544	WN	19393	WN	N7822A	1544	13204	1320402	31454	Orlando, FL	FL	12	Florida	33	14843	1484306	34819	San Juan, PR	PR	72	Puerto Rico	3	1700-1759	1945	0.0	1900-1959	5	0
3	4	2018-02-18	JetBlue Airways	JFK	LAX	1046	384	2475	2018	1	2	18	7	B6	B6	20409	B6	423	B6	20409	B6	N967JB	423	12478	1247805	31703	New York, NY	NY	36	New York	22	12892	1289208	32575	Los Angeles, CA	CA	6	California	91	1000-1059	1410	0.0	1400-1459	10	0
4	5	2018-10-26	Endeavor Air Inc.	ROC	JFK	1850	98	264	2018	4	10	26	5	DL	DL_CODESHARE	19790	DL	5510	9E	20363	9E	N293PQ	5510	14576	1457606	34576	Rochester, NY	NY	36	New York	22	12478	1247805	31703	New York, NY	NY	36	New York	22	1800-1859	2028	0.0	2000-2059	2	0

	Airline	DepTimeBlk	ArrTimeBlk	OriginState	DistanceGroup	Month	DayOfWeek	Flight_Number_Operating_Airline	Operated_or_Branded_Code_Share_Partners	Disruption
3664	Delta Air Lines Inc.	1700-1759	1800-1859	PA	3	4	4	792	DL	0
19869	Air Wisconsin Airlines Corp	2100-2159	2200-2259	IL	1	9	2	3947	UA_CODESHARE	0
14739	SkyWest Airlines Inc.	1400-1459	1500-1559	NY	2	8	7	5130	UA_CODESHARE	0
21303	Southwest Airlines Co.	0800-0859	1200-1259	TX	5	10	5	1364	WN	1
22662	SkyWest Airlines Inc.	0600-0659	0700-0759	FL	4	7	1	3006	AA_CODESHARE	0

ID5059 Knowledge Discovery and Datamining - Assignment 1¶

Import Packages¶

Read in Data¶

Split into Test and Training Sets¶

Data Exploration¶

4.1 Missing Values¶

4.2 Datatypes¶

4.3 Unique Identifiers¶

4.4 Usefulness for the Task¶

Numerical Features¶

5.2 Categorical Features¶

Order¶

Relationship with Response¶

Results of the Timeline Analysis:¶

Results of Data Exploration:¶

Feature Selection¶

Feature Engineering¶

Data Type Conversion¶

Data Wrangling¶

Check for Null Values¶

Encoding¶

Pipeline¶

Standard Scaler¶

Select a Model¶

1. Random Forest¶

Results - Random Forest¶

2. Logistic Regression¶

3. Stochastic Gradient Descent¶